import axios from 'axios';
import { parseStringPromise } from 'xml2js';
import * as cheerio from 'cheerio';
import { chromium, Browser, Page, Request, Response } from 'playwright';

const baseURL = "https://www.woolworths.com.au/sitemap1.xml";


const getSiteMap = async () => {
  const response = await axios.get(baseURL);
  return response.data;
};

const parseSiteMap = async (siteMap: string) => {
  const parsed = await parseStringPromise(siteMap);
  return parsed;
};

const getProductUrls = async () => {
  const siteMap = await getSiteMap();
  const parsedSiteMap = await parseSiteMap(siteMap);
  return parsedSiteMap.urlset.url.map((url: any) => url.loc[0]).filter((url: string) => url.includes('productdetails'));
};



const scrape = async () => {
}
// scrape();

// parse data with url
const parseData = async (url: string) => {
    let browser: Browser | null = null;
  
    try {
      // Launch the browser (headless mode by default)
      browser = await chromium.launch({ headless: false,
        
        args: [
        // '--disable-http2', // Disables HTTP/2
        // '--disable-web-security', // Disable web security (CORS)
        // '--disable-features=IsolateOrigins,site-per-process' // Fix isolation issues
      ] });
      const page: Page = await browser.newPage();
  
      // Listen to all network responses
      page.on('response', async (response: Response) => {
        // console.log('Response URL:', response.url());
        // console.log('Status:', response.status());
        // console.log('Headers:', response.headers());
  
        if (response.url().includes('product/detail')) {
            try {
                const responseBody = await response.text();
                console.log('Response Body:', responseBody);
              } catch (error) {
                console.log('Failed to parse response body:', error);
              }
        }
      });
  
      // Navigate to the target website
      await page.goto(url, { waitUntil: 'networkidle', timeout: 60000 });
  
      // Wait for a few seconds to capture network traffic
      await page.waitForTimeout(5000);
  
    } catch (error) {
      console.error('An error occurred:', error);
    } finally {
      // Close the browser regardless of success or error
      if (browser) {
        await browser.close();
      }
    }
  }
parseData('https://www.woolworths.com.au/shop/productdetails/594774/avocado-fresh');

